package com.lerays.spider.sougou;

import java.sql.SQLException;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.lionsoul.websnail.Spider;
import org.lionsoul.websnail.addrm.Todo;
import org.lionsoul.websnail.schedule.Schedule;
import org.lionsoul.websnail.schedule.TimerItem;
import org.lionsoul.websnail.schedule.TimerTask;

import com.lerays.spider.sougou.hack.SessionFactory;
import com.lerays.spider.sougou.model.Dao;
import com.lerays.spider.sougou.model.SourceBean;
import com.lerays.spider.sougou.utils.Utils;

/**
 * wechat spider startup
 * 
 * @author chenxin<chenxin619315@gmail.com>
 */
public class SougouSpider 
{	
	public static final int NORMAL_INTERVAL 	= 1500;
	public static final int SLOW_INTERVAL 	= 2500;
	
	/**
	 * spider status callback secret key 
	 */
	private String secretKey	= "!wechatspider#~@~&*619315*!";
	private String version		= "15.11.17";
	
	//api url temp
	private String urlTemp = "http://mp.weixin.qq.com/mp/getmasssendmsg?__biz={__biz}&uin={uin}&key={key}&"
			+ "f=json&count=10&devicetype=android-19&lang=zh_CN";
	public static final String EmptySessionErrmsg = "Empty sessions, <a href=\"/admin/spider/wechat/session/add\" class=\"ctrl-link\" style=\"margin-left:5px;\">Add</a> a new session";

	
	private volatile String uin = "Nzk4MjE2NTM2";
	private volatile String key = "1507480b90e51e6355ade5006cc71acb67488861fc9b4184deb6795fa82b050cd2db41ffddaa19b54ca2ce413e8133ef";
	private volatile String where = null;
	private volatile String order = null;
	private volatile String limit = null;
	
	//http request user agent
	private String userAgent = "Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) "
			+ "AppleWebKit/537.51.2 (KHTML, like Gecko) Mobile/11D167 MicroMessenger/5.4 NetType/WIFI";
	
	//maximum retry times for each source bean
	private int maxRetryTimes = 3;
	
	/**
	 * the websnail spider instance 
	 */
	private Spider spider = null;
	private Thread spiderThread = null;
	
	/**
	 * session factory added at 2015-06-15 
	*/
	private SessionFactory sessionFactory = null;
	
	/**
	 * spider shedule added at 2015-07-30 
	*/
	private Schedule schedule = null;
	
	private Dao dao = null;
	
	/**
	 * all the sources, the public number for now 
	 */
	private volatile Map<String, SourceBean> sources = null;
	
	/**
	 * run-time error log 
	 */
	private volatile String lastErrorMsg = null;
	private volatile long lastStartTime = 0;
	private volatile long lastStartDay  = 0;
	private volatile long recentEndTime = 0;
	
	private int failedTaskCount = 0;
	private String[] LOCKS = new String[]{
			new String("a"), 
			new String("b")
	};
	
	/**
	 * task accomplished callback hook 
	 */
	private volatile String taskAccomplishHook = null;
	
	public SougouSpider()
	{
		spider = Spider.costom()
				.setVisitedHandler(null)				//cancle the visitied records
				.setUserAgent(userAgent)
				.setSpiderListener(new SougouSpiderListener(this))
				.setUrlProcessor(new SougouUrlProcessor(this))
				.setPageAnalyzer(new SougouAnalyzer(this))
				.setInterval(NORMAL_INTERVAL)			//set the interval to normal speed
				.build();
		
//		sessionFactory = new SessionFactory();
//		schedule = new Schedule();
	}
	
	/**
	 * init the spider: <br />
	 * 1. load the sources from the remote database server
	 * 2. start the spider 
	 */
	public void start()
	{
Properties pro = Utils.loadConf("db.conf");
                
                //create the dao instance
                String host = pro.getProperty("db.host");
                int port = 3306;
                if ( pro.containsKey("db.port") ) 
                {
                        port = Integer.valueOf(pro.getProperty("db.port"));
                }
                
                String db       = pro.getProperty("db.name");
                String user     = pro.getProperty("db.user");
                String pass = pro.getProperty("db.pass");
                
                boolean unicode = true;
                if ( pro.containsKey("db.unicode") 
                                && pro.getProperty("db.unicode").equals("0") )
                {
                        unicode = false;
                }
                
                String charset  = pro.getProperty("db.charset");
                
                int connectionCacheTime = 60 * 60 * 3;
                if ( pro.containsKey("db.connection.cache") )
                {
                        connectionCacheTime = Integer.valueOf(pro.getProperty("db.connection.cache"));
                }
                
                pro.clear();
                pro = null;
                
                /*System.out.println("db.host: "+host);
                System.out.println("db.port: "+port);
                System.out.println("db.name: "+db);
                System.out.println("db.user: "+user);
                System.out.println("db.pass: "+pass);
                System.out.println("db.unicode: "+unicode);
                System.out.println("db.charset: "+charset);
                System.out.println("db.connection.cache: "+connectionCacheTime);*/
                dao = new Dao(host, port, db, user, pass, unicode, charset);
                dao.setConnectionCacheTime(connectionCacheTime);
	        Todo todo = spider.getTodoHandler();
	        //新浪娱乐
	        todo.add("http://weixin.sogou.com/gzh?openid=oIWsFtzP7MH_dxU6ybGOLA59u-yg&ext=A77vTFKGmhF0iRw7RW6qZJ2LUkKPJNtVi-hQQ3H7R03e2J_rMTZTuQGazDVqAphl");
	        //娱乐圈扒姐
	        todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt5jNYLuSEJI3h64xVRM-nKQ&ext=A77vTFKGmhFksl6_TMvx-4uHhIqAL_rA8cGwRQ_q20Zx2Z2m0VA6exEHKLV5P0wy");
	        //南都娱乐周刊
	        todo.add("http://weixin.sogou.com/gzh?openid=oIWsFtyMaZlfP9ODKwm694OgUKuE&ext=A77vTFKGmhHJ6JWKONAwOmDYM_kVFRJckmIR7v9k_Buyp6FgzQWpRoKPNNcleyOI");
	      //严肃八卦
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt1nr0MKCvpgZ688PR1GUKzs&ext=A77vTFKGmhE5JXYmYfqAc7_nS2f2usFanTkEMYrgq_JBtoyG__i246069w57aQ-0");
                //美美家族
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt4gpfYfLYjc80xVy5G4TrGU&ext=A77vTFKGmhGR5y8nsLoy0dfLEqf1UhvhKt8jSd2jRmu8ejHb1yHUGIlfLTOXEr3j");
                //芭莎娱乐
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFtwFNoysvtA3lD0J2QWuIo5g&ext=A77vTFKGmhGiwhPJMBfGG_EdO6mcQtQilCN9ZmixTaqaM9VAnptwAqto8YBrkiOt");
              //深八影视圈
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFtxEyQME-LjNmo0L0K0XyAgY&ext=A77vTFKGmhFKjXngTgJZ63nhK-7yhEFRmsyV5un9MUMnbJRswofr6iNbNidjKwo2");
                //HKChannel
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFtyFG80BVYNohQmTlyqQhWWQ&ext=A77vTFKGmhFSF3iHX6V0Lo4wDrWxDkZRd2hwk9RenSyHXzCIMDHY8za9gp7QOaCN");
                //凤凰八卦
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt1Gk13mR6kPhL4FQUspSFgc&ext=A77vTFKGmhF5hb6fYcaS96Nz-TG_9akVCzmIxZE7VmrIHEPmF2uvf8BiLhgZTNPb");
              //娱乐八卦
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt5WDWHiyPrGbRFy3moLzqPU&ext=A77vTFKGmhHxKuepkNL7kys_DTLi7oiMb-iWFCxkj-34EAp6Z0d-RNhC96dUmAO9");
                //娱乐圈头条
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt2hqM3Nsx5ZVTqhGoLK9tmg&ext=A77vTFKGmhEBDTHMCGWS-qc3pnP3N7k1OI9b2C8r5cDiTqMILys_rw0WB6EuX14t");
                //养生保健
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFty3bM_xsXmlUSfBBWzswiVE&ext=A77vTFKGmhF43PtDuEUoshZPVXtRCfDF7bv8lVQkvdeX3tDgQbZiKEvts9gdtWSG");
                
              //女性健康频道
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt4iWCNoO68_fZEhxe38BTXg&ext=A77vTFKGmhHL_5HlpNJuMbeO2vGP7OQ83dBueopCUmr8yoUBs40QxFDrg3emnxHg");
                //女子健康顾问
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt5J4Ctip7CaT-piZzmHi4EA&ext=A77vTFKGmhHbLRf82i8SbQGlwWDF8zpqtVZSAj2eOdWDIvVmiyUQY9RzYMu2QfNs");
                //长寿梦
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt17LbKW1EkjC7YFZ80HL0m4&ext=A77vTFKGmhGKWlw1GeAMsl00mfems1I2CdN4sUGRka2xkK2_utwABhwOPrh2stQw");
              //心理咨询师
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt5OAXAv3-qk9vdQrxd6wW0o&ext=A77vTFKGmhGiF_j8L0jQllM6brJWG37EPWRDIL3vVCFFutxrPzXbfNMnDuWH5i1e");
                //女性小知识
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt4YPuXzUbTph-AdIcDHPiQw&ext=A77vTFKGmhGSNd3euXuHXceUt0scGUDgvbsZUcs-TQSS2cQY5cTS3K4iy9Htt0sA");
                //教你瘦身宝典
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFtwUxI4bOfgrlJPZJvZn7Zuk&ext=A77vTFKGmhHJ8pNmQiIzUvZnhfFrydbfn9D52pAWwyZCPxFDy0jKtBq_IS3wNiic");
                
              //健康
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt8eECuEvTO6Q2H6zKy-nsO8&ext=A77vTFKGmhFqQJQHjNeW-22gdUSKUyJsecBtsIUa7ZD3lJ97RlgfjmlEdLlkZRtn");
              //健康天天读
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt6jYCcES-PPSUEP5P0iMGic&ext=A77vTFKGmhFqAxovufASz1pyfDB8dA5k0LqEjbSqH47xllqsyGEh4H00IYo5lWx4");
                //养生之道
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt_XYJo4w5dKHGmrj3d2RhXA&ext=A77vTFKGmhFq6UK_FXcYOOniDtMhFw8mDahSg3KpYku-1-B9zaSpK6ibBcZ5OIbT");
                //养生健康学堂
                todo.add("http://weixin.sogou.com/gzh?openid=oIWsFt-UI6BNoUto7sjLACchBD7Q&ext=A77vTFKGmhGt8rhi_L0xGCBpqgAIayBwdRIPWg4b8MxFeMStt52GgbTALaYD_e5z");

	        //start the main spider thread
		spiderThread = new Thread(new Runnable(){
			@Override
			public void run() 
			{
				spider.start();
			}
		});
		
		spiderThread.start();
	}
	
	/**
	 * Load all the source from the Dao to sources hashmap
	 * yat, you could reinvoke it to update the sources.
	 * 
	 * @throws SQLException 
	 */
	public synchronized void loadSourceFromDb() throws SQLException
	{
		//Load and add all the sources from the dao
		List<SourceBean> list = dao.getAllSource(where, order, limit);
		
		//backup the sources and create a new sources
		Map<String, SourceBean> oldsrc = sources;
		sources = Collections.synchronizedMap(new LinkedHashMap<String, SourceBean>(16, 0.85F));
		
		//mapping them all to the sources
		SourceBean src = null, origin = null;
		Iterator<SourceBean> it = list.iterator();
		while ( it.hasNext() )
		{
			src = it.next();
			sources.put(src.getBiz(), src);
			
			//@Note: follow the lastUpdateTime of the old source record
			if ( oldsrc != null 
					&& (origin = oldsrc.get(src.getBiz())) != null )
			{
				src.setLastUpdateTime(origin.getLastUpdateTime());
			}
		}
		
		//let the gc do its work
		if ( oldsrc != null )
		{
			oldsrc.clear();
			oldsrc = null;
		}
		
		list.clear();
		list = null;
	}
	
	/**
	 * load all the source to the spider#todoHandler
	 *  and start the spider work.
	 * you could reinvoke the method to analysis the sources again
	 * 
	 * this method will make sure all the sources be in the spider#todoHandler
	 */
	public void leadSourceInTodo()
	{
		//check and update the last start time
		// only when the task is commpletely accomplished
		//will we update the lastStartTime
		if ( spider.getTodoHandler().size() == 0 )
		{
			Calendar cal = Calendar.getInstance();
			setLastStartTime(cal.getTimeInMillis() / 1000);			
			cal.set(Calendar.HOUR_OF_DAY, 0);
			cal.set(Calendar.MINUTE, 0);
			cal.set(Calendar.SECOND, 0);
			setLastStartDay(cal.getTimeInMillis() / 1000);
			//reset the interval to normal
			spider.setInterval(NORMAL_INTERVAL);
		}
		
		setRecentEndTime(0);		//clear the recent end time
		setFailedTaskCount(0);
		String key = null;		
		synchronized ( sources ) 
		{
			Iterator<String> it = sources.keySet().iterator();
			Todo todo = spider.getTodoHandler();
			while ( it.hasNext() ) 
			{
				key = it.next();
				sources.get(key).setTryTime(0);		//clear the try times
				todo.add(key);
			}
		}
	}
		
	public String getSecretKey() 
	{
		return secretKey;
	}
	
	public String getVersion() 
	{
		return version;
	}

	/**
	 * get the last database connection time
	 * 
	 * @return	long
	 */
	public long getLastDbConnectTime()
	{
		return dao==null ? 0L : dao.getLastConnectTime();
	}

	/**
	 * get the spider working status
	 * {running | paused}
	 * 
	 * @return	int
	 */
	public int getStatus()
	{
		return spider.getState();
	}
	
	/**
	 * interface to get the origin websnail spider
	 * 
	 * @param	Spider
	 */
	public Spider getSpider()
	{
		return spider;
	}
	
	/**
	 * pause the current spider 
	 */
	public void pause()
	{
		spider.pause();
	}
	
	/**
	 * resume the current spider 
	 */
	public void resume()
	{
		if ( sessionFactory.size() == 0 )
		{
			setLastErrorMsg(EmptySessionErrmsg);
			return;
		}
		
		spider.resume();
	}
	
	public String getUrlTemp() 
	{
		return urlTemp;
	}

	public String getUin() 
	{
		return uin;
	}

	public void setUin(String uin) 
	{
		this.uin = uin;
	}

	public String getKey() 
	{
		return key;
	}

	public void setKey(String key) 
	{
		this.key = key;
		
		//@Note: clear the last error msg for 'key expired'
		if ( getLastErrorMsg() != null 
				&& getLastErrorMsg().indexOf("-3") >-1 )
		{
			setLastErrorMsg(null);
		}
	}
	
	public void setWhere( String where )
	{
		this.where = where;
	}
	
	public String getWhere()
	{
		return where;
	}
	
	public void setOrder( String order )
	{
		this.order = order;
	}
	
	public String getOrder()
	{
		return order;
	}
	
	public String getLimit()
	{
		return limit;
	}
	
	public void setLimit( String limit )
	{
		this.limit = limit;
	}
	
	public String getUserAgent() 
	{
		return spider.getUserAgent();
	}

	public void setUserAgent(String userAgent) 
	{
		spider.setUserAgent(userAgent);
	}

	public int getMaxRetryTimes() 
	{
		return maxRetryTimes;
	}

	public void setMaxRetryTimes(int maxRetryTimes) 
	{
		this.maxRetryTimes = maxRetryTimes;
	}

	public Dao getDao() 
	{
		return dao;
	}

	public String getLastErrorMsg() 
	{
		return lastErrorMsg;
	}

	public void setLastErrorMsg(String lastErrorMsg) 
	{
		this.lastErrorMsg = lastErrorMsg;
	}

	public int getInterval() 
	{
		return spider.getInterval();
	}

	public void setInterval(int interval) 
	{
		spider.setInterval(interval);
	}

	public long getLastStartTime() 
	{
		return lastStartTime;
	}

	public void setLastStartTime(long lastStartTime) 
	{
		this.lastStartTime = lastStartTime;
	}

	public long getLastStartDay() 
	{
		return lastStartDay;
	}

	public void setLastStartDay(long lastStartDay) 
	{
		this.lastStartDay = lastStartDay;
	}

	public long getRecentEndTime() 
	{
		return recentEndTime;
	}

	public void setRecentEndTime(long recentEndTime) 
	{
		synchronized ( LOCKS[0] ) 
		{
			this.recentEndTime = recentEndTime;
		}
	}

	public int getFailedTaskCount() 
	{
		synchronized ( LOCKS[0] ) 
		{
			return failedTaskCount;
		}
	}
	
	public void increaseFailedTaskCount()
	{
		synchronized ( LOCKS[0] ) 
		{
			this.failedTaskCount++;
		}
	}

	public void setFailedTaskCount(int failedTaskCount) 
	{
		this.failedTaskCount = failedTaskCount;
	}

	public String getTaskAccomplishHook() 
	{
		return taskAccomplishHook;
	}

	public void setTaskAccomplishHook(String taskAccomplishHook) 
	{
		this.taskAccomplishHook = taskAccomplishHook;
	}

	public int getSourceSize()
	{
		return sources.size();
	}
	
	public SourceBean getSource( String biz )
	{
		return sources.get(biz);
	}
	
	public Thread getSpiderThread()
	{
		return spiderThread;
	}
	
	public SessionFactory getSessionFactory()
	{
		return sessionFactory;
	}
	
	public Schedule getSchedule()
	{
		return schedule;
	}
	
	/**
	 * shutdown the spider and do some clean up work 
	 */
	public void shutdown()
	{
		if ( sources != null )
		{
			sources.clear();
			sources = null;
		}
		
		//shutdown the spider
		spider.shutdown();
		dao.close();
		sessionFactory.shutdown();
	}
}
